#! /usr/bin/env python
#coding=utf-8
import time
import re
import os

print "###program starts at " + str(time.asctime())
dataFilePath = r"../Data_hfmon/segged/"
fileList = os.listdir(dataFilePath)
segmentHash = {} #segment:df_hash
#df_hash --> timeSliceIdStr:df_t_hash
#df_t_hash --> tweetIDStr:1
windowHash = {} # timeSliceIdStr:tweetNum
psFile = file(dataFilePath + "segment_ps", "w")
for item in fileList:
    if item.find("segged") != 0:
        continue
    print "### Processing " + item
    seggedFile = file(dataFilePath + item)
    tStr = item[len(item)-2:len(item)]
    print "Time window: " + tStr
    tweetNum_t = 0
    while True:

        #[GUA] seggedFile name: * + TimeWindow, format: twitterID, *, twitterText(segment|segment|...), ...
        lineStr = seggedFile.readline()
        lineStr = re.sub(r'\n', " ", lineStr)
        lineStr = lineStr.strip()
        if len(lineStr) <= 0:
            break
        contentArr = lineStr.split("\t")
        tweetIDstr = contentArr[0]
        tweetText = contentArr[2]
        if len(tweetText)*len(tweetIDstr) == 0:
            print "Error: empty id or text: " + tweetIDstr + "#" + tweetText
            exit
        tweetNum_t += 1
        textArr = tweetText.split("|")
        for segment in textArr:
            if segment in segmentHash:
                df_hash = segmentHash[segment]
                if tStr in df_hash:
                    df_t_hash = df_hash[tStr]
                    df_t_hash[tweetIDstr] = 1
                    df_hash[tStr] = df_t_hash
                else:
                    df_t_hash = {}
                    df_t_hash[tweetIDstr] = 1
                    df_hash[tStr] = df_t_hash
                segmentHash[segment] = df_hash
            else:
                df_hash = {}
                df_t_hash = {}
                df_t_hash[tweetIDstr] = 1
                df_hash[tStr] = df_t_hash
                segmentHash[segment] = df_hash
        if tweetNum_t % 100000 == 0:
            print "### " + str(time.asctime()) + " " + str(tweetNum_t) + " tweets are processed!"
    windowHash[tStr] = tweetNum_t
    seggedFile.close()
    print "### " + str(time.asctime()) + " segments in " + item + " are loaded." + str(len(segmentHash))
print "### " + str(len(segmentHash)) + " segments are loaded!"

segNum = 0
for segment in sorted(segmentHash.keys()):
    segNum += 1
    df_hash = segmentHash[segment]
    l = len(df_hash)
    probTemp = 0.0
    for tStr in sorted(df_hash.keys()):
        df_t_hash = df_hash[tStr]
        fst = len(df_t_hash)
        probTemp += (fst*1.0)/windowHash[tStr]
    prob = probTemp/l
    psFile.write(str(prob) + "\t" + segment + "\n")
    if segNum % 100000 == 0:
        print "### " + str(segNum) + " segments are processed at " + str(time.asctime())

print "### segments' ps values are written to " + psFile.name
psFile.close()

print "###program ends at " + str(time.asctime())
